from modelscope import AutoModelForCausalLM, AutoTokenizer
from transformers import TextStreamer

# model_name = "Qwen/Qwen2-1.5B-Instruct"
# model_name = "Qwen/Qwen2-1.5B-Instruct-GPTQ-Int8"
# model_name = "Qwen/Qwen2-1.5B-Instruct-GPTQ-Int4"
model_name = "Qwen/Qwen2.5-0.5B-Instruct"
# model_name = "Qwen/Qwen2-0.5B-Instruct-GGUF"
# model_name = "Qwen/Qwen2-0.5B-Instruct-GPTQ-Int8"
# model_name = "Qwen/Qwen2-0.5B-Instruct-GPTQ-Int4"

model = AutoModelForCausalLM.from_pretrained(
    model_name,
    torch_dtype="auto",
    device_map="auto"
)
tokenizer = AutoTokenizer.from_pretrained(model_name)

prompt = "你是谁"
messages = [
    {"role": "system", "content": "You are Qwen, created by Alibaba Cloud. You are a helpful assistant."},
    {"role": "user", "content": prompt},
]
text = tokenizer.apply_chat_template(
    messages,
    tokenize=False,
    add_generation_prompt=True,
)
model_inputs = tokenizer([text], return_tensors="pt").to(model.device)

streamer = TextStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
generated_ids = model.generate(
    **model_inputs,
    max_new_tokens=512,
    streamer=streamer,
)
# generated_ids = [
#     output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
# ]

# response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
# print(response)
